# coding:utf-8

import sys 

sys.path.append('..')

from pymongo import MongoClient
from pymongo import ASCENDING
from pymongo import DESCENDING
import re, json
from bson.regex import Regex
import traceback
from i_util.logs import LogHandler
import pickle

class SelectMongo():
    def __init__(self, logger, mongo_conf = {}):
        self.logger = logger
        self.mongo_conf = mongo_conf
        self.mongo_host = self.mongo_conf.get('host',None)
        self.mongo_port = int(self.mongo_conf.get('port',0))
        self.mongo_db = self.mongo_conf.get('database',None)
        self.mongo_user = self.mongo_conf.get('username',None)
        self.mongo_passwd = self.mongo_conf.get('password',None)
        self.db = None
        self.__conn = None
        if self.mongo_host and self.mongo_port and self.mongo_db:
            self.__conn = MongoClient(self.mongo_host, self.mongo_port)
            self.db = self.__conn[self.mongo_db]
            if self.mongo_user and self.mongo_passwd:
                self.db.authenticate(self.mongo_user, self.mongo_passwd)

    def __del__(self):
        if self.__conn:
            self.__conn.close()

    def select_by_url_format(self, collection_name, url_format, start=0, limit = -1, extra_filter='{}'):
        try:
            regex = re.compile(url_format, re.IGNORECASE)
            extra_filter = json.loads(extra_filter)
            extra_filter.update({'url': {'$regex': regex}})
            if limit > 0:
                item_cursor = self.db[collection_name].find(extra_filter).skip(start).limit(limit)
            else:
                item_cursor = self.db[collection_name].find(extra_filter).skip(start);
        except Exception, e:
            raise e
        return item_cursor

def main():
    logger = LogHandler('select_webpage'); 
    mongo_conf = { 
        'host': "mongo0",
        'port': 40042,
        'database': "crawl_merge_linkattr",
        'username':"work",
        'password':"haizhi"
    }   
    beanstalk_conf = { 
        "host":"10.25.114.50",
        "port" : 11300,
        "output_tube": "online_download_rsp"
    }   
    processor = SelectMongo(logger, mongo_conf)
    condiction = { 
        'site': 'www.innotree.cn',
        'url_format':'^http://www.innotree.cn/project_extra/detail/',
        'start': 0,
        'limit': -1, 
        'extra_filter': '{"download_time": {"$gt": "2016-11-10 11:30"}}'
    }   
    #results = processor.select_webpage_to_mq(condiction);
    items = processor.select_by_url_format('court.gov.cn', '^http://wenshu.court.gov.cn/CreateContentJS/CreateContentJS.aspx\?DocID=')
    for item in items:
        #link_attr = pickle.loads(item['link_attr'])
        link      = item['url']
        print link

if __name__ == '__main__':
    main()
